#include "genomedatauploader.h"

//create lists of gff and fasta files
//NOTE:  multi-file upload does not work at present
GenomeDataUploader::GenomeDataUploader(QStringList gf, QStringList ff, QObject *parent) :
    gffFiles(gf), fastaFiles(ff), QObject(parent)
{
    //zero gff files have been read, at the start of the uploader
    countGffFilesRead = 0;

    genomeElementCount = 0;
}

//push singleton gff and fasta files onto lists
GenomeDataUploader::GenomeDataUploader(QString gf, QString ff, QObject *parent) :
    QObject(parent)
{
    //-----------APPEND SINGLETON FILES TO LIST--------
    gffFiles.append(gf);
    fastaFiles.append(ff);

    //zero gff files have been read, at the start of the uploader
    countGffFilesRead = 0;
    genomeElementCount = 0;


    //--------INSTANTIATE THE DATABASE---------
    //  Note: later may not use a constant DATA_TYPE, so will need to change this
//    if( DATA_TYPE == PROPRIETARY_DATA )
//    {
//        databaseInterface = new ProprietaryGenomeLocalDatabaseInterface(this);
//    }
//    else if( DATA_TYPE == PUBMED_DATA )
//    {
//        databaseInterface = new PubMedGenomeLocalDatabaseInterface(this);
//    }
}

//begin the genome data upload and processing pipeline
void GenomeDataUploader::process()
{
    qDebug() << "I am in a data uploader thread " << endl;

    //-----------Tell programmer what data processing mode you are in-----------
    if( DATA_TYPE == PROPRIETARY_DATA )
        qDebug() << "Processing proprietary data " << endl;
    else if( DATA_TYPE == PUBMED_DATA )
        qDebug() << "Processing pubmed data " << endl;


    //----------------------------------------------
    //LAUNCH THREADS TO CLUSTER GENES IN FASTA FILES
    processFastaFiles();


}

//------------------STEP 1:  LAUNCH THREADS TO PROCESS FASTA FILES-------------------------
void GenomeDataUploader::processFastaFiles()
{
    if( fastaFiles.size() > 1 )
        appendTogetherFastaFiles();

    //then will run on just one fasta file ... even though there is a for loop here
    //for(int i = 0; i < fastaFiles.size(); i++ )
    //{
        QThread* thread = new QThread;
        FastaClusterGenerator* clusterGenerator = new FastaClusterGenerator(fastaFiles[0]);
        clusterGenerator->moveToThread(thread);
        connect(clusterGenerator, SIGNAL(error(QString)), this, SLOT(errorString(QString)));
        connect(thread, SIGNAL(started()), clusterGenerator, SLOT(process())); //once thread launched, begin processing
        connect(clusterGenerator, SIGNAL(finished()), thread, SLOT(quit()));//quit thread once cluster generator is finished
        connect(clusterGenerator, SIGNAL(finished()), clusterGenerator, SLOT(deleteLater())); //once finsished, delete cluster generator
        connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));//once finished, delete thread
        connect( clusterGenerator, SIGNAL(clusterFileGeneratedSignal(QString)), this, SLOT(clusterFileGeneratedSlot(QString)) );//when cd_hit done, and cluster file generated, catch this file name
        //connect( clusterGenerator, SIGNAL(trackProgress(int)), this, SLOT(updateProgressBar(int))); //am not capturing progress info right now, will eventually
        //connect( clusterGenerator, SIGNAL(finished()), this, SLOT(createContigsSlot())); //not connecting finished to anything- key is generation of right number of cluster files
        thread->start();
    //}
}

void GenomeDataUploader::appendTogetherFastaFiles()
{
    qDebug() << "generating single fasta file from " << fastaFiles.size() << " files" << endl;
    QString outputFastaFile =  QApplication::applicationDirPath().append("/programFiles/dataFiles/").append("allFasta");
    QFile singleFastaFile(outputFastaFile);
    singleFastaFile.open(QIODevice::WriteOnly | QIODevice::Text);
    QTextStream out(&singleFastaFile);
    for(int i = 0; i < fastaFiles.size(); i++)
    {
        QFile fasta( fastaFiles[i] );
        if( !fasta.exists() )
        {
            qDebug() << "fasta file not found." << endl;
            return ;
        }
        if( ! fasta.open(QIODevice::ReadOnly))
        {
            qDebug() << "file " << fastaFiles[i] << " can't be opened." << endl;
            return ;
        }

        QTextStream fastaStream(&fasta);
        QString line;
        do {
            line = fastaStream.readLine(); //read each line
            out << line << endl;


        } while( !fastaStream.atEnd());

        fasta.close();
    }
    singleFastaFile.close();

    fastaFiles.clear();
    fastaFiles.append( outputFastaFile );

    qDebug() << "done: " << fastaFiles.size() << " fasta files" << endl;

}


void GenomeDataUploader::appendTogetherGffFiles()
{
    qDebug() << "generating single fasta file from " << gffFiles.size() << " files" << endl;
    QString outputGffFile =  QApplication::applicationDirPath().append("/programFiles/dataFiles/").append("allGff");
    QFile singleGffFile(outputGffFile);
    singleGffFile.open(QIODevice::WriteOnly | QIODevice::Text);
    QTextStream out(&singleGffFile);
    for(int i = 0; i < gffFiles.size(); i++)
    {
        QFile gff( gffFiles[i] );
        if( !gff.exists() )
        {
            qDebug() << "gff file not found." << endl;
            return ;
        }
        if( ! gff.open(QIODevice::ReadOnly))
        {
            qDebug() << "file " << gffFiles[i] << " can't be opened." << endl;
            return ;
        }

        QTextStream gffStream(&gff);
        QString line;
        QString speciesLine = gffFiles[i];
        int idx = speciesLine.lastIndexOf("/");
        speciesLine.remove(0, idx+1);
        speciesLine.remove(".gff");
        out << "#species: " << speciesLine << endl;//gffFiles[i].remove(".gff").remove( QApplication::applicationDirPath() ) << endl;
        do {
            line = gffStream.readLine(); //read each line
            out << line << endl;


        } while( !gffStream.atEnd());

        gff.close();
    }
    singleGffFile.close();

    gffFiles.clear();
    gffFiles.append( outputGffFile );

    qDebug() << "done: " << gffFiles.size() << " gff files" << endl;

}


//----------------STEP 2: HANDLE GENERATED CLUSTER FILES -------------------
//Called when cluster file generator is done running cd_hit algorthim.
void GenomeDataUploader::clusterFileGeneratedSlot(QString clusterFileName)
{
    clusterFiles << clusterFileName;
    if( clusterFiles.size() == fastaFiles.size() ) //if retrieved all cluster files
    {
        generateClusterHashTable(); //step 3: generate the hashtable for all cluster files

        //step 4: once cluster hash table generated, launch gff processing threads
        processGffFiles();
    }
}

//-----------------------STEP 3: GENERATE HASH TABLE FOR GENEIDs -> CLUSTER IDS----------------
//this function takes all the generated cluster files and creates a hash table associating the
// gene id to the cluster.
//      NOTE!!  I haven't written the pubmed processing stuff yet, but I hope the following will be true in either case:
//              Suppose the geneId in the cluster file is 1234567890 | abcdefghijk.pep or something along those lines
//              the clusterHashKey could be abcdefghijk .  So the geneId should contain the clusterHashId, but may also have
//              extra info that is not in the clusterHashKey.
void GenomeDataUploader::generateClusterHashTable()
{
    qDebug() << "generating cluster hash table " << endl;
    int clusterIdCount = 0;//this is used to keep track of total number of cluster ids
    QString actualClusterId = "Cluster0"; //begin with 'cluster0'
    QString currentClusterId = "Cluster0";//store the cluster id most recently encountered
    //loop through all the cluster files
    for( int i = 0; i < clusterFiles.size(); i++ )
    {
        QFile cluster( clusterFiles[i] );
        if( !cluster.exists() )
        {
            qDebug() << "cluster file not found." << endl;
            return ;
        }
        if( ! cluster.open(QIODevice::ReadOnly))
        {
            qDebug() << "file " << clusterFiles[i] << " can't be opened." << endl;
            return ;
        }
        qDebug() << "file " << clusterFiles[i] << " opened." << endl;

        QTextStream clusterStream(&cluster);

        QString clusterId = ".";
        QString geneId;
        QString line;
        do {
            line = clusterStream.readLine(); //read each line
            //qDebug() << "line = " << line << endl;

            if( line.contains("Cluster") ) //grab cluster ids
            {
                clusterId = line.remove(">").remove(" ");//grab actual cluster id in the file

                //----deal with duplicate cluster id in multiple files potential---
                if( clusterId != currentClusterId ) //if next cluster id encountered
                {
                    currentClusterId = clusterId; //store this cluster id as the new cluster id
                    clusterIdCount++; //increment cluster id count
                }
                clusterId = "Cluster" + QString::number(clusterIdCount);//cluster id to store should be based on count, not value in file
            }
            else //grab gene id
            {
                //QRegExp rx(">.*.pep");//grabs the >geneId.pep  OLD METHOD-- only works for proprietary
                QRegExp rx(">.*");//grabs the >geneId<space>
                int pos = rx.indexIn( line );
                QStringList list = rx.capturedTexts();
                for(int i = 0; i < list.size(); i++)
                {
                    //------PROCESS GENE ID FOR EACH DATA TYPE -----
                    //so that it will be found in the gff file
                    if( DATA_TYPE == PROPRIETARY_DATA )
                    {
                        geneId = list[i].remove(".pep").remove(">").remove("...").remove("at").remove(" ");//I think this will only work for faa ...
                    }
                    else if( DATA_TYPE == PUBMED_DATA )
                    {
                        //need to grab <gi|1234|ref|geneId|...
                        QStringList tokens = list[i].split("|");
                        geneId = tokens[3];//geneId is the 3rd token
                    }

                    geneIdToClusterHashTable.insert(geneId, clusterId); //insert in hashtable
                    //qDebug() << "------ geneId    " << geneId << "   ----  cluster id     " << clusterId << endl;

                }
            }
        }while( !clusterStream.atEnd());

        cluster.close();
    }

    qDebug() << "done generating cluster hash table " << endl;

}

//-----------------------STEP 4: PROCESS GFF FILES TO GET GENOME ELEMENT DATA PACKETS----------------
void GenomeDataUploader::processGffFiles()
{
    appendTogetherGffFiles();

    //------------------------------------------------
    //LAUNCH THREADS TO READ GFF FILES
    for(int i = 0; i < gffFiles.size(); i++)
    {
        QThread* thread = new QThread;
        GffReader* gffReader;
        if( DATA_TYPE == PROPRIETARY_DATA )
        {
            gffReader = new ProprietaryGffReader(gffFiles[i], geneIdToClusterHashTable );
            gffReader->moveToThread(thread);
            connect(gffReader, SIGNAL(error(QString)), this, SLOT(errorString(QString)));
            connect(thread, SIGNAL(started()), gffReader, SLOT(process()));
            connect(gffReader, SIGNAL(finished()), thread, SLOT(quit()));
            connect(gffReader, SIGNAL(finished()), gffReader, SLOT(deleteLater()));
            connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
            connect( gffReader, SIGNAL(dataPacketGenerated(GenomeElementDataPacket*)), this, SLOT(dataPacketGeneratedSlot(GenomeElementDataPacket*)) );
            connect( gffReader, SIGNAL( finished() ), this, SLOT( completedGffFileReadSlot() ));
            //connect( clusterGenerator, SIGNAL(trackProgress(int)), this, SLOT(updateProgressBar(int))); //am not capturing progress info right now, will eventually
            thread->start();
        }
        else if( DATA_TYPE == PUBMED_DATA )
        {
            //ASSUMPTION:  if more than one gff file, name of species is specified and is in the filename
//            bool hasSpecies = false;
//            QString speciesName = "";
//            if( gffFiles.size() > 1 )
//            {
//                hasSpecies = true;
//                int idxOfSlash = gffFiles[i].lastIndexOf('/');
//                QString temp(gffFiles[i]);
//                speciesName = temp.remove(0, idxOfSlash+1);
//                speciesName.remove(".gff");
//                int idxOfLastUnderscore = speciesName.lastIndexOf("_");
//                speciesName.remove(idxOfLastUnderscore, speciesName.size()-idxOfLastUnderscore);
//            }

            //once pubmed reader is created ... insert here
            gffReader = new PubMedGffReader(gffFiles[i], geneIdToClusterHashTable );
            gffReader->moveToThread(thread);
            connect(gffReader, SIGNAL(error(QString)), this, SLOT(errorString(QString)));
            connect(thread, SIGNAL(started()), gffReader, SLOT(process()));
            connect(gffReader, SIGNAL(finished()), thread, SLOT(quit()));
            connect(gffReader, SIGNAL(finished()), gffReader, SLOT(deleteLater()));
            connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater()));
            connect( gffReader, SIGNAL(dataPacketGenerated(GenomeElementDataPacket*)), this, SLOT(dataPacketGeneratedSlot(GenomeElementDataPacket*)) );
            connect( gffReader, SIGNAL( finished() ), this, SLOT( completedGffFileReadSlot() ));
            //connect( clusterGenerator, SIGNAL(trackProgress(int)), this, SLOT(updateProgressBar(int))); //am not capturing progress info right now, will eventually
            thread->start();
        }
    }
}

//-------------STEP 5: HANDLE GENERATED GENOME ELEMENT DATA PACKETS----------
//this function takes generated data packets one by one, as they are created.
//if less than 1000 contigs have been encountered, this genome element will bed to a contig data packet.
//In either case, it also adds the data packets to the database
void GenomeDataUploader::dataPacketGeneratedSlot(GenomeElementDataPacket* dataPacket)
{

//    int numToDisplay = 1000; //only show this many contigs
//    if( dataPacket->getType().compare("CDS") != 0 )//Only show coding sequences
//        return;

//    if( contigDataPacketHash.contains( dataPacket->getParentContigId() ) )
//    {
//        QPointer< ContigDataPacket > c = contigDataPacketHash.value( dataPacket->getParentContigId() );//get it
//        c->addGenomeElement( dataPacket );//and add data packet

//        genomeElementDataPackets << dataPacket; //store data packet in list

//    }
//    else if( contigDataPackets.size() < numToDisplay )//else if allowed to create contig
//    {
//        QPointer< ContigDataPacket > c = new ContigDataPacket( dataPacket->getParentContigId(), dataPacket->getHasSpecies(), dataPacket->getSpecies() );//create contig
//        contigDataPackets << c; //add it to the list
//        //databaseInterface->addContig(c);
//        contigDataPacketHash.insert(dataPacket->getParentContigId(), c);//add it to hash table

//        c->addGenomeElement(dataPacket);//add genome element

//        genomeElementDataPackets << dataPacket; //store data packet in list

//    }
//    else//just add contig to database
//    {

//    }
    int numToDisplay;// = 1000;//1000; //10000000000000;
    if(DISPLAY_MODE == WALL_MODE )
        numToDisplay = 500;
    else
        numToDisplay = 200;
    if( dataPacket->getType().compare("CDS") != 0)//just do coding sequences
        return;
    if( contigDataPackets.size() < numToDisplay )//create the local cache of contigs for display
    {
        if( contigDataPacketHash.contains( dataPacket->getParentContigId()) )//if already encountered this contig
        {
            QPointer< ContigDataPacket > c = contigDataPacketHash.value( dataPacket->getParentContigId() );//get it
            c->addGenomeElement( dataPacket );//and add data packet
        }
        else //else create contig
        {
            QPointer< ContigDataPacket > c = new ContigDataPacket( dataPacket->getParentContigId(), dataPacket->getHasSpecies(), dataPacket->getSpecies() );//create contig
            contigDataPackets << c; //add it to the list
            //databaseInterface->addContig(c);
            contigDataPacketHash.insert(dataPacket->getParentContigId(), c);//add it to hash table

            c->addGenomeElement(dataPacket);//just added this- bug.

            //store curr contig... this is for the else condition below
            currContigDataPacket = c;


        }
        genomeElementDataPackets << dataPacket; //store data packet in list
        genomeElementCount++;
        //databaseInterface->addGenomeElement(dataPacket); //add element to database
    }
    else//if exceeds cache size, just add to database (and have the database delete the object)
    {
        if( dataPacket->getParentContigId().compare( currContigDataPacket->getContigId()) == 0 )
        {
            currContigDataPacket->addGenomeElement(dataPacket);
        }
        else
        {
            emit(contigToCacheSignal(currContigDataPacket) );//will add to db and delete
            currContigDataPacket = new ContigDataPacket( dataPacket->getParentContigId(), dataPacket->getHasSpecies(), dataPacket->getSpecies()  );
            currContigDataPacket->addGenomeElement(dataPacket);
        }
       genomeElementCount++;
            if( genomeElementCount % 1000 == 0 )
                qDebug() << "Signal genome element to cache " << genomeElementCount << endl;
//        emit( genomeElementToCacheSignal(dataPacket) );
        //databaseInterface->addGenomeElement(dataPacket); //add element to database
    }
}

//---------STEP 6: COUNT COMPLETED GFF FILES AND EMIT DATA UPLOAD COMPLETE----------
// sends data packets to main program
void GenomeDataUploader::completedGffFileReadSlot()
{
    qDebug() << "completed read of a gff file " << countGffFilesRead << " " << gffFiles.size() << endl;
    qDebug() << "number of genome elements created: " << genomeElementCount << endl;
    countGffFilesRead++;
    if( countGffFilesRead == gffFiles.size() )
    {
        emit( dataUploadCompleteSignal(genomeElementDataPackets, contigDataPackets, genomeElementCount) );
        emit( finished() );
    }
}

//Slot to catch errors in the fasta cluster generator
void GenomeDataUploader::errorString(QString theError)
{
    qDebug() << theError << endl;

    if( theError.contains("Clustering failed") )
    {
        emit( error(theError) );
    }
}


void GenomeDataUploader::cleanup()
{
    QDir dir;
    dir.remove(QApplication::applicationDirPath().append( "/programFiles/dataFiles/cdHitOut" ));
    dir.remove(QApplication::applicationDirPath().append( "/programFiles/dataFiles/cdHitOut.clstr" ));
    dir.remove(QApplication::applicationDirPath().append( "/programFiles/dataFiles/cdHitOut.bak.clstr" ));
}


